<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<title>Reranker Framework (ReFr): run-model.C Source File</title>

<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="doxygen.css" rel="stylesheet" type="text/css" />

<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/javascript">
  $(document).ready(function() { searchBox.OnSelectItem(0); });
</script>

</head>
<body>
<div id="top"><!-- do not remove this div! -->


<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  
  
  <td style="padding-left: 0.5em;">
   <div id="projectname">Reranker Framework (ReFr)
   
   </div>
   <div id="projectbrief">Reranking framework for structure prediction and discriminative language modeling</div>
  </td>
  
  
  
 </tr>
 </tbody>
</table>
</div>

<!-- Generated by Doxygen 1.7.6.1 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search",false,'Search');
</script>
  <div id="navrow1" class="tabs">
    <ul class="tablist">
      <li><a href="index.html"><span>Main&#160;Page</span></a></li>
      <li><a href="namespaces.html"><span>Namespaces</span></a></li>
      <li><a href="annotated.html"><span>Classes</span></a></li>
      <li class="current"><a href="files.html"><span>Files</span></a></li>
      <li>
        <div id="MSearchBox" class="MSearchBoxInactive">
        <span class="left">
          <img id="MSearchSelect" src="search/mag_sel.png"
               onmouseover="return searchBox.OnSearchSelectShow()"
               onmouseout="return searchBox.OnSearchSelectHide()"
               alt=""/>
          <input type="text" id="MSearchField" value="Search" accesskey="S"
               onfocus="searchBox.OnSearchFieldFocus(true)" 
               onblur="searchBox.OnSearchFieldFocus(false)" 
               onkeyup="searchBox.OnSearchFieldChange(event)"/>
          </span><span class="right">
            <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
          </span>
        </div>
      </li>
    </ul>
  </div>
  <div id="navrow2" class="tabs2">
    <ul class="tablist">
      <li><a href="files.html"><span>File&#160;List</span></a></li>
      <li><a href="globals.html"><span>File&#160;Members</span></a></li>
    </ul>
  </div>
</div>
<div class="header">
  <div class="headertitle">
<div class="title">run-model.C</div>  </div>
</div><!--header-->
<div class="contents">
<a href="run-model_8_c.html">Go to the documentation of this file.</a><div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 <span class="comment">// Copyright 2012, Google Inc.</span>
<a name="l00002"></a>00002 <span class="comment">// All rights reserved.</span>
<a name="l00003"></a>00003 <span class="comment">//</span>
<a name="l00004"></a>00004 <span class="comment">// Redistribution and use in source and binary forms, with or without</span>
<a name="l00005"></a>00005 <span class="comment">// modification, are permitted provided that the following conditions are</span>
<a name="l00006"></a>00006 <span class="comment">// met:</span>
<a name="l00007"></a>00007 <span class="comment">//</span>
<a name="l00008"></a>00008 <span class="comment">//   * Redistributions of source code must retain the above copyright</span>
<a name="l00009"></a>00009 <span class="comment">//     notice, this list of conditions and the following disclaimer.</span>
<a name="l00010"></a>00010 <span class="comment">//   * Redistributions in binary form must reproduce the above</span>
<a name="l00011"></a>00011 <span class="comment">//     copyright notice, this list of conditions and the following disclaimer</span>
<a name="l00012"></a>00012 <span class="comment">//     in the documentation and/or other materials provided with the</span>
<a name="l00013"></a>00013 <span class="comment">//     distribution.</span>
<a name="l00014"></a>00014 <span class="comment">//   * Neither the name of Google Inc. nor the names of its</span>
<a name="l00015"></a>00015 <span class="comment">//     contributors may be used to endorse or promote products derived from</span>
<a name="l00016"></a>00016 <span class="comment">//     this software without specific prior written permission.</span>
<a name="l00017"></a>00017 <span class="comment">//</span>
<a name="l00018"></a>00018 <span class="comment">// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</span>
<a name="l00019"></a>00019 <span class="comment">// &quot;AS IS&quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</span>
<a name="l00020"></a>00020 <span class="comment">// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</span>
<a name="l00021"></a>00021 <span class="comment">// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</span>
<a name="l00022"></a>00022 <span class="comment">// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</span>
<a name="l00023"></a>00023 <span class="comment">// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</span>
<a name="l00024"></a>00024 <span class="comment">// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</span>
<a name="l00025"></a>00025 <span class="comment">// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</span>
<a name="l00026"></a>00026 <span class="comment">// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</span>
<a name="l00027"></a>00027 <span class="comment">// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</span>
<a name="l00028"></a>00028 <span class="comment">// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</span>
<a name="l00029"></a>00029 <span class="comment">// -----------------------------------------------------------------------------</span>
<a name="l00030"></a>00030 <span class="comment">//</span>
<a name="l00031"></a>00031 <span class="comment">//</span>
<a name="l00040"></a>00040 <span class="comment"></span>
<a name="l00041"></a>00041 <span class="preprocessor">#include &lt;iostream&gt;</span>
<a name="l00042"></a>00042 <span class="preprocessor">#include &lt;fstream&gt;</span>
<a name="l00043"></a>00043 <span class="preprocessor">#include &lt;string&gt;</span>
<a name="l00044"></a>00044 <span class="preprocessor">#include &lt;cstdlib&gt;</span>
<a name="l00045"></a>00045 <span class="preprocessor">#include &lt;tr1/memory&gt;</span>
<a name="l00046"></a>00046 <span class="preprocessor">#include &lt;vector&gt;</span>
<a name="l00047"></a>00047 
<a name="l00048"></a>00048 <span class="preprocessor">#include &quot;<a class="code" href="candidate_8_h.html" title="Provides the reranker::Candidate class for representing a candidate hypothesis from an initial model...">candidate.H</a>&quot;</span>
<a name="l00049"></a>00049 <span class="preprocessor">#include &quot;<a class="code" href="candidate-set_8_h.html" title="Class to hold a single training instance for a reranker, which is a set of examples, typically the n-best output of some input process, posibly including a gold-standard feature vector.">candidate-set.H</a>&quot;</span>
<a name="l00050"></a>00050 <span class="preprocessor">#include &quot;<a class="code" href="candidate-set-reader_8_h.html" title="Class for reading streams of training or test instances, where each training or test instance is a re...">candidate-set-reader.H</a>&quot;</span>
<a name="l00051"></a>00051 <span class="preprocessor">#include &quot;<a class="code" href="candidate-set-writer_8_h.html" title="Class for writing streams of training or test instances, where each training or test instance is a re...">candidate-set-writer.H</a>&quot;</span>
<a name="l00052"></a>00052 <span class="preprocessor">#include &quot;<a class="code" href="executive-feature-extractor_8_h.html" title="Provides the reranker::ExecutiveFeatureExtractor class.">executive-feature-extractor.H</a>&quot;</span>
<a name="l00053"></a>00053 <span class="preprocessor">#include &quot;<a class="code" href="model_8_h.html" title="Reranker model interface.">model.H</a>&quot;</span>
<a name="l00054"></a>00054 <span class="preprocessor">#include &quot;<a class="code" href="perceptron-model_8_h.html" title="Provides the reranker::PerceptronModel reranker class.">perceptron-model.H</a>&quot;</span>
<a name="l00055"></a>00055 <span class="preprocessor">#include &quot;<a class="code" href="model-merge-reducer_8_h.html" title="Reducer classes for trainer.">model-merge-reducer.H</a>&quot;</span>
<a name="l00056"></a>00056 <span class="preprocessor">#include &quot;<a class="code" href="model-reader_8_h.html" title="Provides the ModelReader  class, which can create Model  instances from a file.">model-reader.H</a>&quot;</span>
<a name="l00057"></a>00057 <span class="preprocessor">#include &quot;<a class="code" href="model-proto-writer_8_h.html" title="Interface for serializer for reranker::Model instances to ModelMessage instances.">model-proto-writer.H</a>&quot;</span>
<a name="l00058"></a>00058 <span class="preprocessor">#include &quot;<a class="code" href="symbol-table_8_h.html" title="Provides the reranker::Symbols interface as well as the reranker::StaticSymbolTable implementation...">symbol-table.H</a>&quot;</span>
<a name="l00059"></a>00059 
<a name="l00060"></a><a class="code" href="run-model_8_c.html#ad72dbcf6d0153db1b8d8a58001feed83">00060</a> <span class="preprocessor">#define DEBUG 0</span>
<a name="l00061"></a>00061 <span class="preprocessor"></span>
<a name="l00062"></a><a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">00062</a> <span class="preprocessor">#define PROG_NAME &quot;run-model&quot;</span>
<a name="l00063"></a>00063 <span class="preprocessor"></span>
<a name="l00064"></a><a class="code" href="run-model_8_c.html#a532f75d7ae60075d58f59fee8b4ccfaa">00064</a> <span class="preprocessor">#define DEFAULT_MAX_EXAMPLES -1</span>
<a name="l00065"></a><a class="code" href="run-model_8_c.html#a9d6fb0cd899c98a59411c20bf68c3792">00065</a> <span class="preprocessor"></span><span class="preprocessor">#define DEFAULT_MAX_CANDIDATES -1</span>
<a name="l00066"></a><a class="code" href="run-model_8_c.html#a67874ec27386954e2402694a58208146">00066</a> <span class="preprocessor"></span><span class="preprocessor">#define DEFAULT_MODEL_CONFIG &quot;PerceptronModel(name(\&quot;MyPerceptronModel\&quot;))&quot;</span>
<a name="l00067"></a><a class="code" href="run-model_8_c.html#ab69236cfb1640d72af7bc485b4a48f7c">00067</a> <span class="preprocessor"></span><span class="preprocessor">#define DEFAULT_REPORTING_INTERVAL 1000</span>
<a name="l00068"></a><a class="code" href="run-model_8_c.html#a6451af4b3cf2d7cf916f17e23afcd42c">00068</a> <span class="preprocessor"></span><span class="preprocessor">#define DEFAULT_COMPACTIFY_INTERVAL 10000</span>
<a name="l00069"></a><a class="code" href="run-model_8_c.html#aff42509eec1e3179a39054b367fc994d">00069</a> <span class="preprocessor"></span><span class="preprocessor">#define DEFAULT_USE_WEIGHTED_LOSS true</span>
<a name="l00070"></a>00070 <span class="preprocessor"></span>
<a name="l00071"></a>00071 <span class="comment">// We use two levels of macros to get the string version of an int constant.</span>
<a name="l00074"></a><a class="code" href="run-model_8_c.html#afc0fe0597af089c04afb9bc5a6475705">00074</a> <span class="comment"></span><span class="preprocessor">#define XSTR(arg) STR(arg)</span>
<a name="l00075"></a>00075 <span class="preprocessor"></span>
<a name="l00076"></a><a class="code" href="run-model_8_c.html#a878d88fe9bab9d7372961ce77b00fa26">00076</a> <span class="preprocessor">#define STR(arg) #arg</span>
<a name="l00077"></a>00077 <span class="preprocessor"></span>
<a name="l00078"></a>00078 <span class="keyword">using namespace </span>std;
<a name="l00079"></a>00079 <span class="keyword">using namespace </span>std::tr1;
<a name="l00080"></a>00080 <span class="keyword">using namespace </span>reranker;
<a name="l00081"></a>00081 
<a name="l00082"></a><a class="code" href="run-model_8_c.html#ab14fc1012197db1b38cabd8b431160b7">00082</a> <span class="keyword">const</span> <span class="keywordtype">char</span> *<a class="code" href="compile-features_8_c.html#ab14fc1012197db1b38cabd8b431160b7">usage_msg</a>[] = {
<a name="l00083"></a>00083   <span class="stringliteral">&quot;Usage:\n&quot;</span>,
<a name="l00084"></a>00084   <a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">PROG_NAME</a> <span class="stringliteral">&quot; -m|--model-file &lt;model file&gt; [--model-config &lt;model config&gt;]\n&quot;</span>
<a name="l00085"></a>00085   <span class="stringliteral">&quot;\t[-t|--train &lt;training input file&gt;+ [-i &lt;input model file&gt;] [--mapper] ]\n&quot;</span>,
<a name="l00086"></a>00086   <span class="stringliteral">&quot;\t-d|--devtest &lt;devtest input file&gt;+\n&quot;</span>,
<a name="l00087"></a>00087   <span class="stringliteral">&quot;\t[-o|--output &lt;candidate set output file&gt;]\n&quot;</span>,
<a name="l00088"></a>00088   <span class="stringliteral">&quot;\t[-h &lt;hyp output file&gt;] [--scores &lt;score output file&gt;]\n&quot;</span>,
<a name="l00089"></a>00089   <span class="stringliteral">&quot;\t[--train-config &lt;training feature extractor config file&gt;]\n&quot;</span>,
<a name="l00090"></a>00090   <span class="stringliteral">&quot;\t[--dev-config &lt;devtest feature extractor config file&gt;]\n&quot;</span>,
<a name="l00091"></a>00091   <span class="stringliteral">&quot;\t[--compactify-feature-uids]\n&quot;</span>,
<a name="l00092"></a>00092   <span class="stringliteral">&quot;\t[-s|--streaming [--compactify-interval &lt;interval&gt;] ] [-u]\n&quot;</span>,
<a name="l00093"></a>00093   <span class="stringliteral">&quot;\t[--no-base64]\n&quot;</span>,
<a name="l00094"></a>00094   <span class="stringliteral">&quot;\t[--min-epochs &lt;min epochs&gt;] [--max-epochs &lt;max epochs&gt;]\n&quot;</span>,
<a name="l00095"></a>00095   <span class="stringliteral">&quot;\t[--max-examples &lt;max num examples&gt;]\n&quot;</span>,
<a name="l00096"></a>00096   <span class="stringliteral">&quot;\t[--max-candidates &lt;max num candidates&gt;]\n&quot;</span>,
<a name="l00097"></a>00097   <span class="stringliteral">&quot;\t[-r &lt;reporting interval&gt;] [ --use-weighted-loss[=][true|false] ]\n&quot;</span>,
<a name="l00098"></a>00098   <span class="stringliteral">&quot;where\n&quot;</span>,
<a name="l00099"></a>00099   <span class="stringliteral">&quot;\t&lt;model file&gt; is the name of the file to which to write out a\n&quot;</span>,
<a name="l00100"></a>00100   <span class="stringliteral">&quot;\t\tnewly-trained model when training (one or more\n&quot;</span>,
<a name="l00101"></a>00101   <span class="stringliteral">&quot;\t\t&lt;training input file&gt;&#39;s specified), or the name of a file\n&quot;</span>,
<a name="l00102"></a>00102   <span class="stringliteral">&quot;\t\tfrom which to load a serialized model when decoding\n&quot;</span>,
<a name="l00103"></a>00103   <span class="stringliteral">&quot;\t&lt;input model file&gt; is an optional input model file as a starting\n&quot;</span>,
<a name="l00104"></a>00104   <span class="stringliteral">&quot;\t\tmodel when training\n&quot;</span>,
<a name="l00105"></a>00105   <span class="stringliteral">&quot;\t&lt;model config&gt; is the optional configuration string for constructing\n&quot;</span>,
<a name="l00106"></a>00106   <span class="stringliteral">&quot;\t\ta new Model instance\n&quot;</span>,
<a name="l00107"></a>00107   <span class="stringliteral">&quot;\t\t(defaults to \&quot;&quot;</span> <a class="code" href="run-model_8_c.html#a67874ec27386954e2402694a58208146">DEFAULT_MODEL_CONFIG</a> <span class="stringliteral">&quot;\&quot;)\n&quot;</span>,
<a name="l00108"></a>00108   <span class="stringliteral">&quot;\t&lt;training input file&gt; is the name of a stream of serialized\n&quot;</span>,
<a name="l00109"></a>00109   <span class="stringliteral">&quot;\t\tCandidateSet instances, or \&quot;-\&quot; for input from standard input\n&quot;</span>,
<a name="l00110"></a>00110   <span class="stringliteral">&quot;\t--mapper specifies to train a single epoch and output features to\n&quot;</span>,
<a name="l00111"></a>00111   <span class="stringliteral">&quot;\t\tstandard output\n&quot;</span>,
<a name="l00112"></a>00112   <span class="stringliteral">&quot;\t&lt;devtest input file&gt; is the name of a stream of serialized\n&quot;</span>,
<a name="l00113"></a>00113   <span class="stringliteral">&quot;\t\tCandidateSet instances, or \&quot;-\&quot; for input from standard input\n&quot;</span>,
<a name="l00114"></a>00114   <span class="stringliteral">&quot;\t\t(required unless training in mapper mode)\n&quot;</span>,
<a name="l00115"></a>00115   <span class="stringliteral">&quot;\t&lt;candidate set output file&gt; is the name of the file to which to output\n&quot;</span>,
<a name="l00116"></a>00116   <span class="stringliteral">&quot;\t\tcandidate sets that have been scored by the model (in\n&quot;</span>,
<a name="l00117"></a>00117   <span class="stringliteral">&quot;\t\tdecoding mode)\n&quot;</span>,
<a name="l00118"></a>00118   <span class="stringliteral">&quot;\t&lt;training feature extractor config file&gt; is the name of a configuration\n&quot;</span>,
<a name="l00119"></a>00119   <span class="stringliteral">&quot;\t\tfile to be read by the ExecutiveFeatureExtractor instance\n&quot;</span>
<a name="l00120"></a>00120   <span class="stringliteral">&quot;\t\textracting features on training examples\n&quot;</span>,
<a name="l00121"></a>00121   <span class="stringliteral">&quot;\t&lt;devtest feature extractor config file&gt; is the name of a configuration\n&quot;</span>,
<a name="l00122"></a>00122   <span class="stringliteral">&quot;\t\tfile to be read by the ExecutiveFeatureExtractor instance\n&quot;</span>,
<a name="l00123"></a>00123   <span class="stringliteral">&quot;\t\textracting features on devtest examples\n&quot;</span>,
<a name="l00124"></a>00124   <span class="stringliteral">&quot;\t--compactify-feature-uids specifies to re-map all feature uids to the\n&quot;</span>,
<a name="l00125"></a>00125   <span class="stringliteral">&quot;\t\t[0,n-1] interval, where n is the number of non-zero features\n&quot;</span>,
<a name="l00126"></a>00126   <span class="stringliteral">&quot;\t--streaming specifies to train in streaming mode (i.e., do not\n&quot;</span>,
<a name="l00127"></a>00127   <span class="stringliteral">&quot;\t\tread in all training instances into memory)\n&quot;</span>,
<a name="l00128"></a>00128   <span class="stringliteral">&quot;\t--compactify-interval specifies the interval after which to compactify\n&quot;</span>,
<a name="l00129"></a>00129   <span class="stringliteral">&quot;\t\tfeature uid&#39;s and remove unused symbols (only available when\n&quot;</span>,
<a name="l00130"></a>00130   <span class="stringliteral">&quot;\t\ttraining in streaming mode; defaults to &quot;</span>
<a name="l00131"></a>00131   <a class="code" href="run-model_8_c.html#afc0fe0597af089c04afb9bc5a6475705" title="Expands the string value of the specified argument using the STR macro.">XSTR</a>(<a class="code" href="run-model_8_c.html#a6451af4b3cf2d7cf916f17e23afcd42c">DEFAULT_COMPACTIFY_INTERVAL</a>) <span class="stringliteral">&quot;)\n&quot;</span>,
<a name="l00132"></a>00132   <span class="stringliteral">&quot;\t-u specifies that the input files are uncompressed\n&quot;</span>,
<a name="l00133"></a>00133   <span class="stringliteral">&quot;\t--no-base64 specifies not to use base64 encoding/decoding\n&quot;</span>,
<a name="l00134"></a>00134   <span class="stringliteral">&quot;\t--max-examples specifies the maximum number of examples to read from\n&quot;</span>,
<a name="l00135"></a>00135   <span class="stringliteral">&quot;\t\tany input file (defaults to &quot;</span> <a class="code" href="run-model_8_c.html#afc0fe0597af089c04afb9bc5a6475705" title="Expands the string value of the specified argument using the STR macro.">XSTR</a>(<a class="code" href="run-model_8_c.html#a532f75d7ae60075d58f59fee8b4ccfaa">DEFAULT_MAX_EXAMPLES</a>) <span class="stringliteral">&quot;)\n&quot;</span>,
<a name="l00136"></a>00136   <span class="stringliteral">&quot;\t--max-candidates specifies the maximum number of candidates to read\n&quot;</span>,
<a name="l00137"></a>00137   <span class="stringliteral">&quot;\t\tfor any candidate set (defaults to &quot;</span> <a class="code" href="run-model_8_c.html#afc0fe0597af089c04afb9bc5a6475705" title="Expands the string value of the specified argument using the STR macro.">XSTR</a>(<a class="code" href="run-model_8_c.html#a9d6fb0cd899c98a59411c20bf68c3792">DEFAULT_MAX_CANDIDATES</a>) <span class="stringliteral">&quot;)\n&quot;</span>,
<a name="l00138"></a>00138   <span class="stringliteral">&quot;\t-r specifies the interval at which the CandidateSetReader reports how\n&quot;</span>,
<a name="l00139"></a>00139   <span class="stringliteral">&quot;\t\tmany candidate sets it has read (defaults to &quot;</span>
<a name="l00140"></a>00140   <a class="code" href="run-model_8_c.html#afc0fe0597af089c04afb9bc5a6475705" title="Expands the string value of the specified argument using the STR macro.">XSTR</a>(<a class="code" href="run-model_8_c.html#ab69236cfb1640d72af7bc485b4a48f7c">DEFAULT_REPORTING_INTERVAL</a>) <span class="stringliteral">&quot;)\n&quot;</span>,
<a name="l00141"></a>00141   <span class="stringliteral">&quot;\t--use-weighted-loss specifies whether to weight losses on devtest\n&quot;</span>,
<a name="l00142"></a>00142   <span class="stringliteral">&quot;\t\texamples by the number of tokens in the reference, where, e.g.,\n&quot;</span>,
<a name="l00143"></a>00143   <span class="stringliteral">&quot;\t\tweighted loss is appropriate for computing WER, but not BLEU\n&quot;</span>,
<a name="l00144"></a>00144   <span class="stringliteral">&quot;\t\t(defaults to &quot;</span> <a class="code" href="run-model_8_c.html#afc0fe0597af089c04afb9bc5a6475705" title="Expands the string value of the specified argument using the STR macro.">XSTR</a>(<a class="code" href="run-model_8_c.html#aff42509eec1e3179a39054b367fc994d">DEFAULT_USE_WEIGHTED_LOSS</a>) <span class="stringliteral">&quot;)\n&quot;</span>
<a name="l00145"></a>00145 };
<a name="l00146"></a>00146 
<a name="l00149"></a><a class="code" href="run-model_8_c.html#a2ef30c42cbc289d899a8be5d2d8f77d0">00149</a> <span class="keywordtype">void</span> <a class="code" href="compile-features_8_c.html#afe55eae96aed06d16232a3b56fcf1ad3" title="Emits usage message to standard output.">usage</a>() {
<a name="l00150"></a>00150   <span class="keywordtype">int</span> usage_msg_len = <span class="keyword">sizeof</span>(<a class="code" href="compile-features_8_c.html#ab14fc1012197db1b38cabd8b431160b7">usage_msg</a>)/<span class="keyword">sizeof</span>(<span class="keyword">const</span> <span class="keywordtype">char</span> *);
<a name="l00151"></a>00151   <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 0; i &lt; usage_msg_len; ++i) {
<a name="l00152"></a>00152     cout &lt;&lt; <a class="code" href="compile-features_8_c.html#ab14fc1012197db1b38cabd8b431160b7">usage_msg</a>[i];
<a name="l00153"></a>00153   }
<a name="l00154"></a>00154   cout.flush();
<a name="l00155"></a>00155 }
<a name="l00156"></a>00156 
<a name="l00157"></a><a class="code" href="run-model_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">00157</a> <span class="keywordtype">bool</span> <a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(<span class="keywordtype">int</span> argc, <span class="keywordtype">int</span> i, <span class="keywordtype">string</span> err_msg) {
<a name="l00158"></a>00158   <span class="keywordflow">if</span> (i + 1 &gt;= argc) {
<a name="l00159"></a>00159     cerr &lt;&lt; <a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">PROG_NAME</a> &lt;&lt; <span class="stringliteral">&quot;: error: &quot;</span> &lt;&lt; err_msg &lt;&lt; endl;
<a name="l00160"></a>00160     <a class="code" href="compile-features_8_c.html#afe55eae96aed06d16232a3b56fcf1ad3" title="Emits usage message to standard output.">usage</a>();
<a name="l00161"></a>00161     <span class="keywordflow">return</span> <span class="keyword">false</span>;
<a name="l00162"></a>00162   } <span class="keywordflow">else</span> {
<a name="l00163"></a>00163     <span class="keywordflow">return</span> <span class="keyword">true</span>;
<a name="l00164"></a>00164   }
<a name="l00165"></a>00165 }
<a name="l00166"></a>00166 
<a name="l00167"></a><a class="code" href="run-model_8_c.html#a76a3d9ad70569ed55e32127247a6c355">00167</a> <span class="keywordtype">void</span> <a class="code" href="run-model_8_c.html#a76a3d9ad70569ed55e32127247a6c355">read_and_extract_features</a>(<span class="keyword">const</span> vector&lt;string&gt; &amp;files,
<a name="l00168"></a>00168                                <a class="code" href="classreranker_1_1_candidate_set_reader.html" title="A class for reading streams of training or test instances, where each training or test instance is a ...">CandidateSetReader</a> &amp;csr,
<a name="l00169"></a>00169                                <span class="keywordtype">bool</span> compressed,
<a name="l00170"></a>00170                                <span class="keywordtype">bool</span> use_base64,
<a name="l00171"></a>00171                                <a class="code" href="classreranker_1_1_executive_feature_extractor.html" title="This class is like a regular FeatureExtractor, but has been promoted to the executive level and thus ...">ExecutiveFeatureExtractor</a> &amp;efe,
<a name="l00172"></a>00172                                vector&lt;shared_ptr&lt;CandidateSet&gt; &gt; &amp;examples) {
<a name="l00173"></a>00173   <span class="keywordtype">bool</span> reset_counters = <span class="keyword">true</span>;
<a name="l00174"></a>00174   <span class="keywordflow">for</span> (vector&lt;string&gt;::const_iterator file_it = files.begin();
<a name="l00175"></a>00175        file_it != files.end();
<a name="l00176"></a>00176        ++file_it) {
<a name="l00177"></a>00177     csr.<a class="code" href="classreranker_1_1_candidate_set_reader.html#afa8d2312498eb123c00796dbc2dd087a" title="Reads a stream of CandidateSet instances from the specified file or from standard input...">Read</a>(*file_it, compressed, use_base64, reset_counters, examples);
<a name="l00178"></a>00178   }
<a name="l00179"></a>00179   <span class="comment">// Extract features for CandidateSet instances in situ.</span>
<a name="l00180"></a>00180   <span class="keywordflow">for</span> (vector&lt;shared_ptr&lt;CandidateSet&gt; &gt;::iterator it = examples.begin();
<a name="l00181"></a>00181        it != examples.end();
<a name="l00182"></a>00182        ++it) {
<a name="l00183"></a>00183     efe.<a class="code" href="classreranker_1_1_executive_feature_extractor.html#ae825fd69b80d636cb12d95e39a1c3e20" title="Extracts features for the specified CandidateSet using the suite of FeatureExtractor instances specif...">Extract</a>(*(*it));
<a name="l00184"></a>00184   }
<a name="l00185"></a>00185 }
<a name="l00186"></a>00186 
<a name="l00187"></a>00187 <span class="keywordtype">int</span>
<a name="l00188"></a><a class="code" href="run-model_8_c.html#a3c04138a5bfe5d72780bb7e82a18e627">00188</a> <a class="code" href="candidate-set-proto-reader-test_8_c.html#a3c04138a5bfe5d72780bb7e82a18e627">main</a>(<span class="keywordtype">int</span> argc, <span class="keywordtype">char</span> **argv) {
<a name="l00189"></a>00189   <span class="comment">// Required parameters.</span>
<a name="l00190"></a>00190   <span class="keywordtype">string</span> model_file;
<a name="l00191"></a>00191   <span class="keywordtype">string</span> input_model_file;
<a name="l00192"></a>00192   <span class="keywordtype">string</span> model_config = <a class="code" href="run-model_8_c.html#a67874ec27386954e2402694a58208146">DEFAULT_MODEL_CONFIG</a>;
<a name="l00193"></a>00193   vector&lt;string&gt; training_files;
<a name="l00194"></a>00194   vector&lt;string&gt; devtest_files;
<a name="l00195"></a>00195   <span class="keywordtype">bool</span> mapper_mode = <span class="keyword">false</span>;
<a name="l00196"></a>00196   <span class="keywordtype">string</span> output_file;
<a name="l00197"></a>00197   <span class="keywordtype">string</span> hyp_output_file;
<a name="l00198"></a>00198   <span class="keywordtype">string</span> score_output_file;
<a name="l00199"></a>00199   <span class="keywordtype">string</span> training_feature_extractor_config_file;
<a name="l00200"></a>00200   <span class="keywordtype">string</span> devtest_feature_extractor_config_file;
<a name="l00201"></a>00201   <span class="keywordtype">bool</span> compressed = <span class="keyword">true</span>;
<a name="l00202"></a>00202   <span class="keywordtype">bool</span> use_base64 = <span class="keyword">true</span>;
<a name="l00203"></a>00203   <span class="keywordtype">bool</span> streaming = <span class="keyword">false</span>;
<a name="l00204"></a>00204   <span class="keywordtype">bool</span> use_weighted_loss = <a class="code" href="run-model_8_c.html#aff42509eec1e3179a39054b367fc994d">DEFAULT_USE_WEIGHTED_LOSS</a>;
<a name="l00205"></a>00205   <span class="keywordtype">string</span> use_weighted_loss_arg_prefix = <span class="stringliteral">&quot;--use-weighted-loss&quot;</span>;
<a name="l00206"></a>00206   <span class="keywordtype">size_t</span> use_weighted_loss_arg_prefix_len =
<a name="l00207"></a>00207       use_weighted_loss_arg_prefix.length();
<a name="l00208"></a>00208   <span class="keywordtype">bool</span> compactify_feature_uids = <span class="keyword">false</span>;
<a name="l00209"></a>00209   <span class="keywordtype">int</span> compactify_interval = <a class="code" href="run-model_8_c.html#a6451af4b3cf2d7cf916f17e23afcd42c">DEFAULT_COMPACTIFY_INTERVAL</a>;
<a name="l00210"></a>00210   <span class="keywordtype">int</span> min_epochs = -1;
<a name="l00211"></a>00211   <span class="keywordtype">int</span> max_epochs = -1;
<a name="l00212"></a>00212   <span class="keywordtype">int</span> max_examples = <a class="code" href="run-model_8_c.html#a532f75d7ae60075d58f59fee8b4ccfaa">DEFAULT_MAX_EXAMPLES</a>;
<a name="l00213"></a>00213   <span class="keywordtype">int</span> max_candidates = <a class="code" href="run-model_8_c.html#a9d6fb0cd899c98a59411c20bf68c3792">DEFAULT_MAX_CANDIDATES</a>;
<a name="l00214"></a>00214   <span class="keywordtype">int</span> reporting_interval = <a class="code" href="run-model_8_c.html#ab69236cfb1640d72af7bc485b4a48f7c">DEFAULT_REPORTING_INTERVAL</a>;
<a name="l00215"></a>00215 
<a name="l00216"></a>00216   <span class="comment">// Process options.  The majority of code in this file is devoted to this.</span>
<a name="l00217"></a>00217   <span class="keywordflow">for</span> (<span class="keywordtype">int</span> i = 1; i &lt; argc; ++i) {
<a name="l00218"></a>00218     <span class="keywordtype">string</span> arg = argv[i];
<a name="l00219"></a>00219     <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-m&quot;</span> || arg == <span class="stringliteral">&quot;-model&quot;</span> || arg == <span class="stringliteral">&quot;--model&quot;</span>) {
<a name="l00220"></a>00220       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no model file specified with &quot;</span>) + arg;
<a name="l00221"></a>00221       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00222"></a>00222         <span class="keywordflow">return</span> -1;
<a name="l00223"></a>00223       }
<a name="l00224"></a>00224       model_file = argv[++i];
<a name="l00225"></a>00225     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-i&quot;</span> || arg == <span class="stringliteral">&quot;--i&quot;</span>) {
<a name="l00226"></a>00226       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no input model file specified with &quot;</span>) + arg;
<a name="l00227"></a>00227       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00228"></a>00228         <span class="keywordflow">return</span> -1;
<a name="l00229"></a>00229       }
<a name="l00230"></a>00230       input_model_file = argv[++i];      
<a name="l00231"></a>00231     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-model-config&quot;</span> || arg == <span class="stringliteral">&quot;--model-config&quot;</span>) {
<a name="l00232"></a>00232       <span class="keywordtype">string</span> err_msg =
<a name="l00233"></a>00233           string(<span class="stringliteral">&quot;no model configuration string specified with &quot;</span>) + arg;
<a name="l00234"></a>00234       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00235"></a>00235         <span class="keywordflow">return</span> -1;
<a name="l00236"></a>00236       }
<a name="l00237"></a>00237       model_config = argv[++i];
<a name="l00238"></a>00238     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-t&quot;</span> || arg == <span class="stringliteral">&quot;-train&quot;</span> || arg == <span class="stringliteral">&quot;--train&quot;</span>) {
<a name="l00239"></a>00239       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no input files specified with &quot;</span>) + arg;
<a name="l00240"></a>00240       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00241"></a>00241         <span class="keywordflow">return</span> -1;
<a name="l00242"></a>00242       }
<a name="l00243"></a>00243       <span class="comment">// Keep reading args until next option or until no more args.</span>
<a name="l00244"></a>00244       ++i;
<a name="l00245"></a>00245       <span class="keywordflow">for</span> ( ; i &lt; argc; ++i) {
<a name="l00246"></a>00246         <span class="keywordflow">if</span> (argv[i][0] == <span class="charliteral">&#39;-&#39;</span> &amp;&amp; strlen(argv[i]) &gt; 1) {
<a name="l00247"></a>00247           --i;
<a name="l00248"></a>00248           <span class="keywordflow">break</span>;
<a name="l00249"></a>00249         }
<a name="l00250"></a>00250         training_files.push_back(argv[i]);
<a name="l00251"></a>00251       }
<a name="l00252"></a>00252     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-mapper&quot;</span> || arg == <span class="stringliteral">&quot;--mapper&quot;</span>) {
<a name="l00253"></a>00253       mapper_mode = <span class="keyword">true</span>;
<a name="l00254"></a>00254     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-d&quot;</span> || arg == <span class="stringliteral">&quot;-devtest&quot;</span> || arg == <span class="stringliteral">&quot;--devtest&quot;</span>) {
<a name="l00255"></a>00255       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no input files specified with &quot;</span>) + arg;
<a name="l00256"></a>00256       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00257"></a>00257         <span class="keywordflow">return</span> -1;
<a name="l00258"></a>00258       }
<a name="l00259"></a>00259       <span class="comment">// Keep reading args until next option or until no more args.</span>
<a name="l00260"></a>00260       ++i;
<a name="l00261"></a>00261       <span class="keywordflow">for</span> ( ; i &lt; argc; ++i) {
<a name="l00262"></a>00262         <span class="keywordflow">if</span> (argv[i][0] == <span class="charliteral">&#39;-&#39;</span>) {
<a name="l00263"></a>00263           --i;
<a name="l00264"></a>00264           <span class="keywordflow">break</span>;
<a name="l00265"></a>00265         }
<a name="l00266"></a>00266         devtest_files.push_back(argv[i]);
<a name="l00267"></a>00267       }
<a name="l00268"></a>00268     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-o&quot;</span> || arg == <span class="stringliteral">&quot;-output&quot;</span> || arg == <span class="stringliteral">&quot;--output&quot;</span>) {
<a name="l00269"></a>00269       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no output file specified with &quot;</span>) + arg;
<a name="l00270"></a>00270       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00271"></a>00271         <span class="keywordflow">return</span> -1;
<a name="l00272"></a>00272       }
<a name="l00273"></a>00273       output_file = argv[++i];
<a name="l00274"></a>00274     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-h&quot;</span>) {
<a name="l00275"></a>00275       <span class="keywordtype">string</span> err_msg =
<a name="l00276"></a>00276           string(<span class="stringliteral">&quot;no hypothesis output file specified with &quot;</span>) + arg;
<a name="l00277"></a>00277       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00278"></a>00278         <span class="keywordflow">return</span> -1;
<a name="l00279"></a>00279       }
<a name="l00280"></a>00280       hyp_output_file = argv[++i];
<a name="l00281"></a>00281     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-scores&quot;</span> || arg == <span class="stringliteral">&quot;--scores&quot;</span>) {
<a name="l00282"></a>00282       <span class="keywordtype">string</span> err_msg =
<a name="l00283"></a>00283           string(<span class="stringliteral">&quot;no score output file specified with &quot;</span>) + arg;
<a name="l00284"></a>00284       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00285"></a>00285         <span class="keywordflow">return</span> -1;
<a name="l00286"></a>00286       }
<a name="l00287"></a>00287       score_output_file = argv[++i];
<a name="l00288"></a>00288     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-train-config&quot;</span> || arg == <span class="stringliteral">&quot;--train-config&quot;</span>) {
<a name="l00289"></a>00289       <span class="keywordtype">string</span> err_msg =
<a name="l00290"></a>00290           string(<span class="stringliteral">&quot;no feature extractor config file specified with &quot;</span>) + arg;
<a name="l00291"></a>00291       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00292"></a>00292         <span class="keywordflow">return</span> -1;
<a name="l00293"></a>00293       }
<a name="l00294"></a>00294       training_feature_extractor_config_file = argv[++i];
<a name="l00295"></a>00295     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-dev-config&quot;</span> || arg == <span class="stringliteral">&quot;--dev-config&quot;</span>) {
<a name="l00296"></a>00296       <span class="keywordtype">string</span> err_msg =
<a name="l00297"></a>00297           string(<span class="stringliteral">&quot;no feature extractor config file specified with &quot;</span>) + arg;
<a name="l00298"></a>00298       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00299"></a>00299         <span class="keywordflow">return</span> -1;
<a name="l00300"></a>00300       }
<a name="l00301"></a>00301       devtest_feature_extractor_config_file = argv[++i];
<a name="l00302"></a>00302     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-compactify-feature-uids&quot;</span> ||
<a name="l00303"></a>00303                arg == <span class="stringliteral">&quot;--compactify-feature-uids&quot;</span>) {
<a name="l00304"></a>00304       compactify_feature_uids = <span class="keyword">true</span>;
<a name="l00305"></a>00305     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-s&quot;</span> || arg == <span class="stringliteral">&quot;-streaming&quot;</span> || arg == <span class="stringliteral">&quot;--streaming&quot;</span>) {
<a name="l00306"></a>00306       streaming = <span class="keyword">true</span>;
<a name="l00307"></a>00307     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;--compactify-interval&quot;</span>) {
<a name="l00308"></a>00308       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no interval specified with &quot;</span>) + arg;
<a name="l00309"></a>00309       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00310"></a>00310         <span class="keywordflow">return</span> -1;
<a name="l00311"></a>00311       }
<a name="l00312"></a>00312       compactify_interval = atoi(argv[++i]);
<a name="l00313"></a>00313     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-u&quot;</span>) {
<a name="l00314"></a>00314       compressed = <span class="keyword">false</span>;
<a name="l00315"></a>00315     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;--no-base64&quot;</span>) {
<a name="l00316"></a>00316       use_base64 = <span class="keyword">false</span>;
<a name="l00317"></a>00317     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-min-epochs&quot;</span> || arg == <span class="stringliteral">&quot;--min-epochs&quot;</span>) {
<a name="l00318"></a>00318       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no arg specified with &quot;</span>) + arg;
<a name="l00319"></a>00319       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00320"></a>00320         <span class="keywordflow">return</span> -1;
<a name="l00321"></a>00321       }
<a name="l00322"></a>00322       min_epochs = atoi(argv[++i]);
<a name="l00323"></a>00323     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-max-epochs&quot;</span> || arg == <span class="stringliteral">&quot;--max-epochs&quot;</span>) {
<a name="l00324"></a>00324       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no arg specified with &quot;</span>) + arg;
<a name="l00325"></a>00325       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00326"></a>00326         <span class="keywordflow">return</span> -1;
<a name="l00327"></a>00327       }
<a name="l00328"></a>00328       max_epochs = atoi(argv[++i]);
<a name="l00329"></a>00329     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-max-examples&quot;</span> || arg == <span class="stringliteral">&quot;--max-examples&quot;</span>) {
<a name="l00330"></a>00330       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no arg specified with &quot;</span>) + arg;
<a name="l00331"></a>00331       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00332"></a>00332         <span class="keywordflow">return</span> -1;
<a name="l00333"></a>00333       }
<a name="l00334"></a>00334       max_examples = atoi(argv[++i]);
<a name="l00335"></a>00335     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-max-candidates&quot;</span> || arg == <span class="stringliteral">&quot;--max-candidates&quot;</span>) {
<a name="l00336"></a>00336       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no arg specified with &quot;</span>) + arg;
<a name="l00337"></a>00337       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00338"></a>00338         <span class="keywordflow">return</span> -1;
<a name="l00339"></a>00339       }
<a name="l00340"></a>00340       max_candidates = atoi(argv[++i]);
<a name="l00341"></a>00341     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg == <span class="stringliteral">&quot;-r&quot;</span>) {
<a name="l00342"></a>00342       <span class="keywordtype">string</span> err_msg = string(<span class="stringliteral">&quot;no arg specified with &quot;</span>) + arg;
<a name="l00343"></a>00343       <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00344"></a>00344         <span class="keywordflow">return</span> -1;
<a name="l00345"></a>00345       }
<a name="l00346"></a>00346       reporting_interval = atoi(argv[++i]);
<a name="l00347"></a>00347     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg.substr(0, use_weighted_loss_arg_prefix_len) ==
<a name="l00348"></a>00348                use_weighted_loss_arg_prefix) {
<a name="l00349"></a>00349       <span class="keywordtype">string</span> use_weighted_loss_str;
<a name="l00350"></a>00350       <span class="keywordflow">if</span> (arg.length() &gt; use_weighted_loss_arg_prefix_len &amp;&amp;
<a name="l00351"></a>00351           arg[use_weighted_loss_arg_prefix_len] == <span class="charliteral">&#39;=&#39;</span>) {
<a name="l00352"></a>00352         use_weighted_loss_str =
<a name="l00353"></a>00353             arg.substr(use_weighted_loss_arg_prefix_len + 1);
<a name="l00354"></a>00354       } <span class="keywordflow">else</span> {
<a name="l00355"></a>00355         <span class="keywordtype">string</span> err_msg =
<a name="l00356"></a>00356             string(<span class="stringliteral">&quot;no \&quot;true\&quot; or \&quot;false\&quot; arg specified with &quot;</span>) + arg;
<a name="l00357"></a>00357         <span class="keywordflow">if</span> (!<a class="code" href="compile-features_8_c.html#a69a0862c537d5d29e5f898d21b069ab4">check_for_required_arg</a>(argc, i, err_msg)) {
<a name="l00358"></a>00358           <span class="keywordflow">return</span> -1;
<a name="l00359"></a>00359         }
<a name="l00360"></a>00360         use_weighted_loss_str = argv[++i];
<a name="l00361"></a>00361       }
<a name="l00362"></a>00362       <span class="keywordflow">if</span> (use_weighted_loss_str != <span class="stringliteral">&quot;true&quot;</span> &amp;&amp;
<a name="l00363"></a>00363           use_weighted_loss_str != <span class="stringliteral">&quot;false&quot;</span>) {
<a name="l00364"></a>00364         cerr &lt;&lt; <a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">PROG_NAME</a> &lt;&lt; <span class="stringliteral">&quot;: error: must specify \&quot;true\&quot; or \&quot;false\&quot;&quot;</span>
<a name="l00365"></a>00365              &lt;&lt; <span class="stringliteral">&quot; with --use-weighted-loss&quot;</span> &lt;&lt; endl;
<a name="l00366"></a>00366         <a class="code" href="compile-features_8_c.html#afe55eae96aed06d16232a3b56fcf1ad3" title="Emits usage message to standard output.">usage</a>();
<a name="l00367"></a>00367         <span class="keywordflow">return</span> -1;
<a name="l00368"></a>00368       }
<a name="l00369"></a>00369       <span class="keywordflow">if</span> (use_weighted_loss_str != <span class="stringliteral">&quot;true&quot;</span>) {
<a name="l00370"></a>00370         use_weighted_loss = <span class="keyword">false</span>;
<a name="l00371"></a>00371       }
<a name="l00372"></a>00372     } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (arg.size() &gt; 0 &amp;&amp; arg[0] == <span class="charliteral">&#39;-&#39;</span>) {
<a name="l00373"></a>00373       cerr &lt;&lt; <a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">PROG_NAME</a> &lt;&lt; <span class="stringliteral">&quot;: error: unrecognized option: &quot;</span> &lt;&lt; arg &lt;&lt; endl;
<a name="l00374"></a>00374       <a class="code" href="compile-features_8_c.html#afe55eae96aed06d16232a3b56fcf1ad3" title="Emits usage message to standard output.">usage</a>();
<a name="l00375"></a>00375       <span class="keywordflow">return</span> -1;
<a name="l00376"></a>00376     }
<a name="l00377"></a>00377   }
<a name="l00378"></a>00378 
<a name="l00379"></a>00379   <span class="keywordtype">bool</span> training = training_files.size() &gt; 0;
<a name="l00380"></a>00380 
<a name="l00381"></a>00381   <span class="comment">// Check that user specified required args.</span>
<a name="l00382"></a>00382   <span class="keywordflow">if</span> (model_file == <span class="stringliteral">&quot;&quot;</span>) {
<a name="l00383"></a>00383     cerr &lt;&lt; <a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">PROG_NAME</a> &lt;&lt; <span class="stringliteral">&quot;: error: must specify model file&quot;</span> &lt;&lt; endl;
<a name="l00384"></a>00384     <a class="code" href="compile-features_8_c.html#afe55eae96aed06d16232a3b56fcf1ad3" title="Emits usage message to standard output.">usage</a>();
<a name="l00385"></a>00385     <span class="keywordflow">return</span> -1;
<a name="l00386"></a>00386   }
<a name="l00387"></a>00387   <span class="keywordflow">if</span> (!mapper_mode &amp;&amp; devtest_files.size() == 0) {
<a name="l00388"></a>00388     cerr &lt;&lt; <a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">PROG_NAME</a> &lt;&lt; <span class="stringliteral">&quot;: error: must specify devtest input files when &quot;</span>
<a name="l00389"></a>00389          &lt;&lt; <span class="stringliteral">&quot;not in mapper mode&quot;</span> &lt;&lt; endl;
<a name="l00390"></a>00390     <a class="code" href="compile-features_8_c.html#afe55eae96aed06d16232a3b56fcf1ad3" title="Emits usage message to standard output.">usage</a>();
<a name="l00391"></a>00391     <span class="keywordflow">return</span> -1;
<a name="l00392"></a>00392   }
<a name="l00393"></a>00393   <span class="keywordflow">if</span> (output_file != <span class="stringliteral">&quot;&quot;</span> &amp;&amp; training) {
<a name="l00394"></a>00394     cerr &lt;&lt; <a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">PROG_NAME</a> &lt;&lt; <span class="stringliteral">&quot;: error: cannot specify output file when training&quot;</span>
<a name="l00395"></a>00395          &lt;&lt; endl;
<a name="l00396"></a>00396     <a class="code" href="compile-features_8_c.html#afe55eae96aed06d16232a3b56fcf1ad3" title="Emits usage message to standard output.">usage</a>();
<a name="l00397"></a>00397     <span class="keywordflow">return</span> -1;
<a name="l00398"></a>00398   }
<a name="l00399"></a>00399   <span class="keywordflow">if</span> (hyp_output_file != <span class="stringliteral">&quot;&quot;</span> &amp;&amp; training) {
<a name="l00400"></a>00400     cerr &lt;&lt; <a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">PROG_NAME</a>
<a name="l00401"></a>00401          &lt;&lt; <span class="stringliteral">&quot;: error: cannot specify hypothesis output file when training&quot;</span>
<a name="l00402"></a>00402          &lt;&lt; endl;
<a name="l00403"></a>00403     <a class="code" href="compile-features_8_c.html#afe55eae96aed06d16232a3b56fcf1ad3" title="Emits usage message to standard output.">usage</a>();
<a name="l00404"></a>00404     <span class="keywordflow">return</span> -1;
<a name="l00405"></a>00405   }
<a name="l00406"></a>00406   <span class="keywordtype">bool</span> reading_from_stdin = <span class="keyword">false</span>;
<a name="l00407"></a>00407   <span class="keywordflow">for</span> (vector&lt;string&gt;::const_iterator training_file_it = training_files.begin();
<a name="l00408"></a>00408        training_file_it != training_files.end();
<a name="l00409"></a>00409        ++training_file_it) {
<a name="l00410"></a>00410     <span class="keywordflow">if</span> (*training_file_it == <span class="stringliteral">&quot;-&quot;</span>) {
<a name="l00411"></a>00411       reading_from_stdin = <span class="keyword">true</span>;
<a name="l00412"></a>00412       <span class="keywordflow">break</span>;
<a name="l00413"></a>00413     }
<a name="l00414"></a>00414   }
<a name="l00415"></a>00415   <span class="keywordflow">if</span> (training_files.size() &gt; 1 &amp;&amp; reading_from_stdin) {
<a name="l00416"></a>00416     cerr &lt;&lt; <a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">PROG_NAME</a> &lt;&lt; <span class="stringliteral">&quot;: error: cannot read from standard input and &quot;</span>
<a name="l00417"></a>00417          &lt;&lt; <span class="stringliteral">&quot;specify other training files&quot;</span> &lt;&lt; endl;
<a name="l00418"></a>00418     <a class="code" href="compile-features_8_c.html#afe55eae96aed06d16232a3b56fcf1ad3" title="Emits usage message to standard output.">usage</a>();
<a name="l00419"></a>00419     <span class="keywordflow">return</span> -1;
<a name="l00420"></a>00420   }
<a name="l00421"></a>00421   <span class="keywordflow">if</span> (!training &amp;&amp; input_model_file != <span class="stringliteral">&quot;&quot;</span>) {
<a name="l00422"></a>00422     cerr &lt;&lt; <a class="code" href="run-model_8_c.html#a44493dbd21a529ec6fdb2da73a35ac70">PROG_NAME</a> &lt;&lt; <span class="stringliteral">&quot;: error: can only specify &lt;input model file&gt; &quot;</span>
<a name="l00423"></a>00423          &lt;&lt; <span class="stringliteral">&quot;when in training mode&quot;</span> &lt;&lt; endl;
<a name="l00424"></a>00424     <a class="code" href="compile-features_8_c.html#afe55eae96aed06d16232a3b56fcf1ad3" title="Emits usage message to standard output.">usage</a>();
<a name="l00425"></a>00425     <span class="keywordflow">return</span> -1;
<a name="l00426"></a>00426   }
<a name="l00427"></a>00427 
<a name="l00428"></a>00428   <span class="comment">// Now, we finally get to the meat of the code for this executable.</span>
<a name="l00429"></a>00429   <a class="code" href="classreranker_1_1_executive_feature_extractor.html" title="This class is like a regular FeatureExtractor, but has been promoted to the executive level and thus ...">ExecutiveFeatureExtractor</a> training_efe;
<a name="l00430"></a>00430   <span class="keywordflow">if</span> (training_feature_extractor_config_file != <span class="stringliteral">&quot;&quot;</span>) {
<a name="l00431"></a>00431     training_efe.<a class="code" href="classreranker_1_1_executive_feature_extractor.html#ab970de3165f9979a995011c520b0813f" title="Initializes this instance from the feature extractor specifications in the specified file...">Init</a>(training_feature_extractor_config_file);
<a name="l00432"></a>00432   }
<a name="l00433"></a>00433   <a class="code" href="classreranker_1_1_executive_feature_extractor.html" title="This class is like a regular FeatureExtractor, but has been promoted to the executive level and thus ...">ExecutiveFeatureExtractor</a> devtest_efe;
<a name="l00434"></a>00434   <span class="keywordflow">if</span> (devtest_feature_extractor_config_file != <span class="stringliteral">&quot;&quot;</span>) {
<a name="l00435"></a>00435     devtest_efe.<a class="code" href="classreranker_1_1_executive_feature_extractor.html#ab970de3165f9979a995011c520b0813f" title="Initializes this instance from the feature extractor specifications in the specified file...">Init</a>(devtest_feature_extractor_config_file);
<a name="l00436"></a>00436   }
<a name="l00437"></a>00437 
<a name="l00438"></a>00438   <a class="code" href="classreranker_1_1_candidate_set_reader.html" title="A class for reading streams of training or test instances, where each training or test instance is a ...">CandidateSetReader</a> csr(max_examples, max_candidates, reporting_interval);
<a name="l00439"></a>00439   csr.<a class="code" href="classreranker_1_1_candidate_set_reader.html#a46df40e5ad98bdee0bb0a552512349d5" title="Sets the verbosity of this reader (mostly for debugging purposes).">set_verbosity</a>(1);
<a name="l00440"></a>00440 
<a name="l00441"></a>00441   shared_ptr&lt;Model&gt; model;
<a name="l00442"></a>00442   <a class="code" href="classreranker_1_1_factory.html">Factory&lt;Model&gt;</a> model_factory;
<a name="l00443"></a>00443 
<a name="l00444"></a>00444   <span class="keywordflow">if</span> (!training || input_model_file != <span class="stringliteral">&quot;&quot;</span>) {
<a name="l00445"></a>00445     <span class="comment">// We&#39;re here because we&#39;re not training, or else we are training and</span>
<a name="l00446"></a>00446     <span class="comment">// the user specified an input model file.</span>
<a name="l00447"></a>00447     <span class="keywordtype">string</span> model_file_to_load = training ? input_model_file : model_file;
<a name="l00448"></a>00448 
<a name="l00449"></a>00449     <a class="code" href="classreranker_1_1_model_reader.html" title="Knows how to create Model instances that have been serialized to a file.">ModelReader</a> model_reader(1);
<a name="l00450"></a>00450     model = model_reader.<a class="code" href="classreranker_1_1_model_reader.html#a55f682f65a7c93ecbbc8089108f7edf2">Read</a>(model_file_to_load, compressed, use_base64);
<a name="l00451"></a>00451   } <span class="keywordflow">else</span> {
<a name="l00452"></a>00452     <span class="comment">// First, see if model_config is the name of a file.</span>
<a name="l00453"></a>00453     ifstream model_config_is(model_config.c_str());
<a name="l00454"></a>00454     <span class="keywordflow">if</span> (model_config_is) {
<a name="l00455"></a>00455       cerr &lt;&lt; <span class="stringliteral">&quot;Reading model config from file \&quot;&quot;</span> &lt;&lt; model_config &lt;&lt; <span class="stringliteral">&quot;\&quot;.&quot;</span>
<a name="l00456"></a>00456            &lt;&lt; endl;
<a name="l00457"></a>00457     }
<a name="l00458"></a>00458 
<a name="l00459"></a>00459     <a class="code" href="classreranker_1_1_stream_tokenizer.html" title="A simple class for tokenizing a stream of tokens for the formally specified language used to construc...">StreamTokenizer</a> *st = model_config_is.good() ?
<a name="l00460"></a>00460         <span class="keyword">new</span> <a class="code" href="classreranker_1_1_stream_tokenizer.html" title="A simple class for tokenizing a stream of tokens for the formally specified language used to construc...">StreamTokenizer</a>(model_config_is) :
<a name="l00461"></a>00461         <span class="keyword">new</span> <a class="code" href="classreranker_1_1_stream_tokenizer.html" title="A simple class for tokenizing a stream of tokens for the formally specified language used to construc...">StreamTokenizer</a>(model_config);
<a name="l00462"></a>00462     model = model_factory.<a class="code" href="classreranker_1_1_factory.html#a68117013334b03c94fc525e62080311f" title="Dynamically creates an object, whose type and initialization are contained in a specification string...">CreateOrDie</a>(*st);
<a name="l00463"></a>00463     <span class="keyword">delete</span> st;
<a name="l00464"></a>00464   }
<a name="l00465"></a>00465   <span class="keywordflow">if</span> (model.get() == NULL) {
<a name="l00466"></a>00466     <span class="keywordflow">return</span> -1;
<a name="l00467"></a>00467   }
<a name="l00468"></a>00468 
<a name="l00469"></a>00469   <a class="code" href="classreranker_1_1_factory.html" title="Factory for dynamically created instance of the specified type.">Factory&lt;ModelProtoWriter&gt;</a> proto_writer_factory;
<a name="l00470"></a>00470   shared_ptr&lt;ModelProtoWriter&gt; model_writer =
<a name="l00471"></a>00471       proto_writer_factory.<a class="code" href="classreranker_1_1_factory.html#a68117013334b03c94fc525e62080311f" title="Dynamically creates an object, whose type and initialization are contained in a specification string...">CreateOrDie</a>(model-&gt;proto_writer_spec(),
<a name="l00472"></a>00472                                        <span class="stringliteral">&quot;model proto writer&quot;</span>);
<a name="l00473"></a>00473   <span class="keywordflow">if</span> (model_writer.get() == NULL) {
<a name="l00474"></a>00474     <span class="keywordflow">return</span> -1;
<a name="l00475"></a>00475   }
<a name="l00476"></a>00476 
<a name="l00477"></a>00477   <span class="keywordflow">if</span> (!mapper_mode) {
<a name="l00478"></a>00478     model-&gt;set_end_of_epoch_hook(<span class="keyword">new</span> <a class="code" href="classreranker_1_1_end_of_epoch_model_writer.html" title="An end-of-epoch hook for writing out the best model so far to file after each epoch (if the best mode...">EndOfEpochModelWriter</a>(model_file,
<a name="l00479"></a>00479                                                            model_writer,
<a name="l00480"></a>00480                                                            compressed,
<a name="l00481"></a>00481                                                            use_base64));
<a name="l00482"></a>00482   }
<a name="l00483"></a>00483   model-&gt;set_use_weighted_loss(use_weighted_loss);
<a name="l00484"></a>00484   model-&gt;set_min_epochs(min_epochs);
<a name="l00485"></a>00485   model-&gt;set_max_epochs(max_epochs);
<a name="l00486"></a>00486 
<a name="l00487"></a>00487   vector&lt;shared_ptr&lt;CandidateSet&gt; &gt; training_examples;
<a name="l00488"></a>00488   vector&lt;shared_ptr&lt;CandidateSet&gt; &gt; devtest_examples;
<a name="l00489"></a>00489   <span class="keywordflow">if</span> (!streaming &amp;&amp; !mapper_mode) {
<a name="l00490"></a>00490     cerr &lt;&lt; <span class="stringliteral">&quot;Loading devtest examples.&quot;</span> &lt;&lt; endl;
<a name="l00491"></a>00491     <a class="code" href="run-model_8_c.html#a76a3d9ad70569ed55e32127247a6c355">read_and_extract_features</a>(devtest_files, csr, compressed, use_base64,
<a name="l00492"></a>00492                               devtest_efe, devtest_examples);
<a name="l00493"></a>00493     <span class="keywordflow">if</span> (devtest_examples.size() == 0) {
<a name="l00494"></a>00494       cerr &lt;&lt; <span class="stringliteral">&quot;Could not read any devtest examples.  Exiting.&quot;</span> &lt;&lt; endl;
<a name="l00495"></a>00495       <span class="keywordflow">return</span> -1;
<a name="l00496"></a>00496     }
<a name="l00497"></a>00497   }
<a name="l00498"></a>00498 
<a name="l00499"></a>00499   <span class="keyword">typedef</span> <a class="code" href="classreranker_1_1_collection_candidate_set_iterator.html" title="An implementation of the CandidateSetIterator interface that is backed by an arbitrary C++ collection...">CollectionCandidateSetIterator&lt;vector&lt;shared_ptr&lt;CandidateSet&gt;</a> &gt; &gt;
<a name="l00500"></a>00500       CandidateSetVectorIt;
<a name="l00501"></a>00501 
<a name="l00502"></a>00502   <a class="code" href="classreranker_1_1_candidate_set_iterator.html" title="An interface specifying iteration over CandidateSet instances, using Java-style semantics (sorry...">CandidateSetIterator</a> *training_it;
<a name="l00503"></a>00503   <a class="code" href="classreranker_1_1_candidate_set_iterator.html" title="An interface specifying iteration over CandidateSet instances, using Java-style semantics (sorry...">CandidateSetIterator</a> *devtest_it;
<a name="l00504"></a>00504 
<a name="l00505"></a>00505   <span class="keywordflow">if</span> (training_files.size() &gt; 0) {
<a name="l00506"></a>00506     cerr &lt;&lt; <span class="stringliteral">&quot;Training.&quot;</span> &lt;&lt; endl;
<a name="l00507"></a>00507     <span class="keywordflow">if</span> (streaming) {
<a name="l00508"></a>00508       training_it = <span class="keyword">new</span> <a class="code" href="classreranker_1_1_multi_file_candidate_set_iterator.html" title="An implementation of the CandidateSetIterator interface that iterates over CandidateSet instances tha...">MultiFileCandidateSetIterator</a>(training_files,
<a name="l00509"></a>00509                                                       &amp;training_efe,
<a name="l00510"></a>00510                                                       max_examples,
<a name="l00511"></a>00511                                                       max_candidates,
<a name="l00512"></a>00512                                                       reporting_interval,
<a name="l00513"></a>00513                                                       1,
<a name="l00514"></a>00514                                                       compressed, use_base64);
<a name="l00515"></a>00515       devtest_it = <span class="keyword">new</span> <a class="code" href="classreranker_1_1_multi_file_candidate_set_iterator.html" title="An implementation of the CandidateSetIterator interface that iterates over CandidateSet instances tha...">MultiFileCandidateSetIterator</a>(devtest_files,
<a name="l00516"></a>00516                                                      &amp;devtest_efe,
<a name="l00517"></a>00517                                                      max_examples,
<a name="l00518"></a>00518                                                      max_candidates,
<a name="l00519"></a>00519                                                      reporting_interval,
<a name="l00520"></a>00520                                                      1,
<a name="l00521"></a>00521                                                      compressed, use_base64);
<a name="l00522"></a>00522       <span class="comment">// TODO(dbikel): Make sure to add setter method to Model and</span>
<a name="l00523"></a>00523       <span class="comment">//               PerceptronModel to tell model to invoke its</span>
<a name="l00524"></a>00524       <span class="comment">//               CompactifyFeatureUids method after a specified</span>
<a name="l00525"></a>00525       <span class="comment">//               interval.  This new setter method should only</span>
<a name="l00526"></a>00526       <span class="comment">//               be invoked here, when in streaming mode.</span>
<a name="l00527"></a>00527     } <span class="keywordflow">else</span> {
<a name="l00528"></a>00528       <span class="comment">// Regular, in-memory, non-streaming training.</span>
<a name="l00529"></a>00529       <a class="code" href="run-model_8_c.html#a76a3d9ad70569ed55e32127247a6c355">read_and_extract_features</a>(training_files, csr, compressed, use_base64,
<a name="l00530"></a>00530                                 training_efe, training_examples);
<a name="l00531"></a>00531       <span class="keywordflow">if</span> (training_examples.size() == 0) {
<a name="l00532"></a>00532         cerr &lt;&lt; <span class="stringliteral">&quot;Could not read any training examples from training files.&quot;</span>
<a name="l00533"></a>00533              &lt;&lt; <span class="stringliteral">&quot;  Exiting.&quot;</span> &lt;&lt; endl;
<a name="l00534"></a>00534         <span class="keywordflow">return</span> -1;
<a name="l00535"></a>00535       }
<a name="l00536"></a>00536       csr.<a class="code" href="classreranker_1_1_candidate_set_reader.html#a7288ad426c48448041df81e2be1595bf" title="Invokes CandidateSetProtoReader::ClearStrings on the internal CandidateSetProtoReader instance...">ClearStrings</a>();
<a name="l00537"></a>00537 
<a name="l00538"></a>00538       training_it = <span class="keyword">new</span> CandidateSetVectorIt(training_examples);
<a name="l00539"></a>00539       devtest_it = <span class="keyword">new</span> CandidateSetVectorIt(devtest_examples);
<a name="l00540"></a>00540     }
<a name="l00541"></a>00541 
<a name="l00542"></a>00542     <span class="keywordflow">if</span> (mapper_mode) {
<a name="l00543"></a>00543       <span class="comment">// In mapper mode, train a single epoch, then write out features</span>
<a name="l00544"></a>00544       <span class="comment">// to stdout, and serialize model.</span>
<a name="l00545"></a>00545       model-&gt;NewEpoch();
<a name="l00546"></a>00546       model-&gt;TrainOneEpoch(*training_it);
<a name="l00547"></a>00547     } <span class="keywordflow">else</span> {
<a name="l00548"></a>00548       model-&gt;Train(*training_it, *devtest_it);
<a name="l00549"></a>00549       <span class="keyword">delete</span> training_it;
<a name="l00550"></a>00550       <span class="keyword">delete</span> devtest_it;
<a name="l00551"></a>00551     }
<a name="l00552"></a>00552 
<a name="l00553"></a>00553     <span class="keywordflow">if</span> (compactify_feature_uids) {
<a name="l00554"></a>00554       cerr &lt;&lt; <span class="stringliteral">&quot;Compactifying feature uid&#39;s...&quot;</span>;
<a name="l00555"></a>00555       cerr.flush();
<a name="l00556"></a>00556       model-&gt;CompactifyFeatureUids();
<a name="l00557"></a>00557       cerr &lt;&lt; <span class="stringliteral">&quot;done.&quot;</span> &lt;&lt; endl;
<a name="l00558"></a>00558     }
<a name="l00559"></a>00559 
<a name="l00560"></a>00560     <span class="comment">// Serialize model.</span>
<a name="l00561"></a>00561     cerr &lt;&lt; <span class="stringliteral">&quot;Writing out model to file \&quot;&quot;</span> &lt;&lt; model_file &lt;&lt; <span class="stringliteral">&quot;\&quot;...&quot;</span>;
<a name="l00562"></a>00562     cerr.flush();
<a name="l00563"></a>00563     confusion_learning::ModelMessage model_message;
<a name="l00564"></a>00564     model_writer-&gt;Write(model.get(), &amp;model_message, <span class="keyword">false</span>);
<a name="l00565"></a>00565 
<a name="l00566"></a>00566     ConfusionProtoIO* proto_writer;
<a name="l00567"></a>00567     <span class="keywordflow">if</span> (mapper_mode) {
<a name="l00568"></a>00568       cerr &lt;&lt; <span class="stringliteral">&quot;Writing ModelMessage (without features) and FeatureMessage &quot;</span>
<a name="l00569"></a>00569            &lt;&lt; <span class="stringliteral">&quot;instances to standard output.&quot;</span> &lt;&lt; endl;
<a name="l00570"></a>00570       proto_writer = <span class="keyword">new</span> ConfusionProtoIO(model_file, ConfusionProtoIO::WRITESTD,
<a name="l00571"></a>00571                                           <span class="keyword">false</span>, use_base64);
<a name="l00572"></a>00572       cout &lt;&lt; ModelInfoReducer::kModelMessageFeatureName &lt;&lt; <span class="stringliteral">&quot;\t&quot;</span>;
<a name="l00573"></a>00573     } <span class="keywordflow">else</span> {
<a name="l00574"></a>00574       proto_writer = <span class="keyword">new</span> ConfusionProtoIO(model_file, ConfusionProtoIO::WRITE,
<a name="l00575"></a>00575                                           compressed, use_base64);
<a name="l00576"></a>00576     }
<a name="l00577"></a>00577     proto_writer-&gt;Write(model_message);
<a name="l00578"></a>00578     <span class="comment">// Write out features.</span>
<a name="l00579"></a>00579     <span class="keywordtype">bool</span> output_best_epoch = !mapper_mode;
<a name="l00580"></a>00580     <span class="keywordtype">bool</span> output_key = mapper_mode;
<a name="l00581"></a>00581     model_writer-&gt;WriteFeatures(model.get(), 
<a name="l00582"></a>00582                                 *(proto_writer-&gt;outputstream()),
<a name="l00583"></a>00583                                 output_best_epoch,
<a name="l00584"></a>00584                                 model-&gt;num_training_errors(),
<a name="l00585"></a>00585                                 output_key);
<a name="l00586"></a>00586     <span class="keyword">delete</span> proto_writer;
<a name="l00587"></a>00587     cerr &lt;&lt; <span class="stringliteral">&quot;done.&quot;</span> &lt;&lt; endl;
<a name="l00588"></a>00588   } <span class="keywordflow">else</span> {
<a name="l00589"></a>00589     CandidateSetVectorIt devtest_examples_it(devtest_examples);
<a name="l00590"></a>00590     model-&gt;NewEpoch(); <span class="comment">// sets epoch to 0</span>
<a name="l00591"></a>00591     model-&gt;Evaluate(devtest_examples_it);
<a name="l00592"></a>00592 
<a name="l00593"></a>00593     <span class="keywordflow">if</span> (output_file != <span class="stringliteral">&quot;&quot;</span>) {
<a name="l00594"></a>00594       <a class="code" href="classreranker_1_1_candidate_set_writer.html" title="A class for writing streams of training or test instances, where each training or test instance is a ...">CandidateSetWriter</a> csw;
<a name="l00595"></a>00595       csw.<a class="code" href="classreranker_1_1_candidate_set_writer.html#af87c88a58e8279fa2548e8b09c710693" title="Sets the verbosity of this writer (mostly for debugging purposes).">set_verbosity</a>(1);
<a name="l00596"></a>00596       csw.<a class="code" href="classreranker_1_1_candidate_set_writer.html#a1909c6197ee6e708018b8fb47c381675" title="Writes a stream of CandidateSet instances to the specified file or to standard output.">Write</a>(devtest_examples, output_file, compressed, use_base64);
<a name="l00597"></a>00597     }
<a name="l00598"></a>00598     <span class="keywordtype">bool</span> output_hyps = hyp_output_file != <span class="stringliteral">&quot;&quot;</span>;
<a name="l00599"></a>00599     <span class="keywordtype">bool</span> output_scores = score_output_file != <span class="stringliteral">&quot;&quot;</span>;
<a name="l00600"></a>00600     <span class="keywordflow">if</span> (output_hyps || output_scores) {
<a name="l00601"></a>00601       ofstream hyp_os(hyp_output_file.c_str());
<a name="l00602"></a>00602       ofstream score_os(score_output_file.c_str());
<a name="l00603"></a>00603       devtest_examples_it.Reset();
<a name="l00604"></a>00604       <span class="keywordflow">while</span> (devtest_examples_it.HasNext()) {
<a name="l00605"></a>00605         <a class="code" href="classreranker_1_1_candidate_set.html" title="A class to hold a set of candidates, either for training or test.">CandidateSet</a> &amp;candidate_set = devtest_examples_it.Next();
<a name="l00606"></a>00606         <span class="keywordflow">if</span> (output_hyps) {
<a name="l00607"></a>00607           hyp_os &lt;&lt; candidate_set.<a class="code" href="classreranker_1_1_candidate_set.html#ae2d8533d6d2d3cd10a2c23a8660eefc3">GetBestScoring</a>().<a class="code" href="classreranker_1_1_candidate.html#abb17414a33453509e2b7cfaf8be02baa" title="Returns the raw data (typically the sentence) for this candidate.">raw_data</a>() &lt;&lt; <span class="stringliteral">&quot;\n&quot;</span>;
<a name="l00608"></a>00608         }
<a name="l00609"></a>00609         <span class="keywordflow">if</span> (output_scores) {
<a name="l00610"></a>00610           <span class="keywordflow">for</span> (<a class="code" href="classreranker_1_1_candidate_set.html#aa907c4710944e91361cef4d465116544">CandidateSet::const_iterator</a> cand_it = candidate_set.<a class="code" href="classreranker_1_1_candidate_set.html#a2a6e17ba0eff32f7c48435c9456c7fe7">begin</a>();
<a name="l00611"></a>00611                cand_it != candidate_set.<a class="code" href="classreranker_1_1_candidate_set.html#aab273fd7ef076bc5221e0d099c38b8aa">end</a>();
<a name="l00612"></a>00612                ++cand_it) {
<a name="l00613"></a>00613             score_os &lt;&lt; (*cand_it)-&gt;score() &lt;&lt; <span class="stringliteral">&quot;\n&quot;</span>;            
<a name="l00614"></a>00614           }
<a name="l00615"></a>00615         }
<a name="l00616"></a>00616       }
<a name="l00617"></a>00617       <span class="keywordflow">if</span> (output_hyps) {
<a name="l00618"></a>00618         hyp_os.flush();
<a name="l00619"></a>00619       }
<a name="l00620"></a>00620       <span class="keywordflow">if</span> (output_scores) {
<a name="l00621"></a>00621         score_os.flush();
<a name="l00622"></a>00622       }
<a name="l00623"></a>00623     }
<a name="l00624"></a>00624   }
<a name="l00625"></a>00625   <a class="code" href="namespacereranker.html#aad9b478cf30e8884067b0f4edefefb76" title="A free-floating function (within the reranker namespace) that frees statically allocated objects...">TearDown</a>();
<a name="l00626"></a>00626   google::protobuf::ShutdownProtobufLibrary();
<a name="l00627"></a>00627 }
<a name="l00628"></a>00628 
</pre></div></div><!-- contents -->
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark">&#160;</span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark">&#160;</span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark">&#160;</span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark">&#160;</span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark">&#160;</span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark">&#160;</span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark">&#160;</span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark">&#160;</span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark">&#160;</span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark">&#160;</span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark">&#160;</span>Defines</a></div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>



<hr class="footer"/><address class="footer"><small>
Generated on Tue Apr 9 2013 11:56:29 for Reranker Framework (ReFr) by &#160;<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/>
</a> 1.7.6.1
</small></address>

</body>
</html>
